{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0-alpha0\n",
      "sys.version_info(major=3, minor=7, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.0.3\n",
      "numpy 1.16.2\n",
      "pandas 0.24.2\n",
      "sklearn 0.20.3\n",
      "tensorflow 2.0.0-alpha0\n",
      "tensorflow.python.keras.api._v2.keras 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow import keras\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, tf, keras:\n",
    "    print(module.__name__, module.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "value: \"machine learning\"\n",
      "value: \"cc150\"\n",
      "\n",
      "value: 15.5\n",
      "value: 9.5\n",
      "value: 7.0\n",
      "value: 8.0\n",
      "\n",
      "value: 42\n",
      "\n",
      "feature {\n",
      "  key: \"age\"\n",
      "  value {\n",
      "    int64_list {\n",
      "      value: 42\n",
      "    }\n",
      "  }\n",
      "}\n",
      "feature {\n",
      "  key: \"favorite_books\"\n",
      "  value {\n",
      "    bytes_list {\n",
      "      value: \"machine learning\"\n",
      "      value: \"cc150\"\n",
      "    }\n",
      "  }\n",
      "}\n",
      "feature {\n",
      "  key: \"hours\"\n",
      "  value {\n",
      "    float_list {\n",
      "      value: 15.5\n",
      "      value: 9.5\n",
      "      value: 7.0\n",
      "      value: 8.0\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# tfrecord 文件格式\n",
    "# -> tf.train.Example\n",
    "#    -> tf.train.Features -> {\"key\": tf.train.Feature}\n",
    "#       -> tf.train.Feature -> tf.train.ByteList/FloatList/Int64List\n",
    "\n",
    "favorite_books = [name.encode('utf-8')\n",
    "                  for name in [\"machine learning\", \"cc150\"]]\n",
    "favorite_books_bytelist = tf.train.BytesList(value = favorite_books)\n",
    "print(favorite_books_bytelist)\n",
    "\n",
    "hours_floatlist = tf.train.FloatList(value = [15.5, 9.5, 7.0, 8.0])\n",
    "print(hours_floatlist)\n",
    "\n",
    "age_int64list = tf.train.Int64List(value = [42])\n",
    "print(age_int64list)\n",
    "\n",
    "features = tf.train.Features(\n",
    "    feature = {\n",
    "        \"favorite_books\": tf.train.Feature(\n",
    "            bytes_list = favorite_books_bytelist),\n",
    "        \"hours\": tf.train.Feature(\n",
    "            float_list = hours_floatlist),\n",
    "        \"age\": tf.train.Feature(int64_list = age_int64list),\n",
    "    }\n",
    ")\n",
    "print(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features {\n",
      "  feature {\n",
      "    key: \"age\"\n",
      "    value {\n",
      "      int64_list {\n",
      "        value: 42\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "  feature {\n",
      "    key: \"favorite_books\"\n",
      "    value {\n",
      "      bytes_list {\n",
      "        value: \"machine learning\"\n",
      "        value: \"cc150\"\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "  feature {\n",
      "    key: \"hours\"\n",
      "    value {\n",
      "      float_list {\n",
      "        value: 15.5\n",
      "        value: 9.5\n",
      "        value: 7.0\n",
      "        value: 8.0\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n",
      "b'\\n\\\\\\n\\x1d\\n\\x05hours\\x12\\x14\\x12\\x12\\n\\x10\\x00\\x00xA\\x00\\x00\\x18A\\x00\\x00\\xe0@\\x00\\x00\\x00A\\n-\\n\\x0efavorite_books\\x12\\x1b\\n\\x19\\n\\x10machine learning\\n\\x05cc150\\n\\x0c\\n\\x03age\\x12\\x05\\x1a\\x03\\n\\x01*'\n"
     ]
    }
   ],
   "source": [
    "example = tf.train.Example(features=features)\n",
    "print(example)\n",
    "\n",
    "serialized_example = example.SerializeToString()\n",
    "print(serialized_example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = 'tfrecord_basic'\n",
    "if not os.path.exists(output_dir):\n",
    "    os.mkdir(output_dir)\n",
    "filename = \"test.tfrecords\"\n",
    "filename_fullpath = os.path.join(output_dir, filename)\n",
    "with tf.io.TFRecordWriter(filename_fullpath) as writer:\n",
    "    for i in range(3):\n",
    "        writer.write(serialized_example)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(b'\\n\\\\\\n\\x1d\\n\\x05hours\\x12\\x14\\x12\\x12\\n\\x10\\x00\\x00xA\\x00\\x00\\x18A\\x00\\x00\\xe0@\\x00\\x00\\x00A\\n-\\n\\x0efavorite_books\\x12\\x1b\\n\\x19\\n\\x10machine learning\\n\\x05cc150\\n\\x0c\\n\\x03age\\x12\\x05\\x1a\\x03\\n\\x01*', shape=(), dtype=string)\n",
      "tf.Tensor(b'\\n\\\\\\n\\x1d\\n\\x05hours\\x12\\x14\\x12\\x12\\n\\x10\\x00\\x00xA\\x00\\x00\\x18A\\x00\\x00\\xe0@\\x00\\x00\\x00A\\n-\\n\\x0efavorite_books\\x12\\x1b\\n\\x19\\n\\x10machine learning\\n\\x05cc150\\n\\x0c\\n\\x03age\\x12\\x05\\x1a\\x03\\n\\x01*', shape=(), dtype=string)\n",
      "tf.Tensor(b'\\n\\\\\\n\\x1d\\n\\x05hours\\x12\\x14\\x12\\x12\\n\\x10\\x00\\x00xA\\x00\\x00\\x18A\\x00\\x00\\xe0@\\x00\\x00\\x00A\\n-\\n\\x0efavorite_books\\x12\\x1b\\n\\x19\\n\\x10machine learning\\n\\x05cc150\\n\\x0c\\n\\x03age\\x12\\x05\\x1a\\x03\\n\\x01*', shape=(), dtype=string)\n"
     ]
    }
   ],
   "source": [
    "dataset = tf.data.TFRecordDataset([filename_fullpath])\n",
    "for serialized_example_tensor in dataset:\n",
    "    print(serialized_example_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "machine learning\n",
      "cc150\n",
      "machine learning\n",
      "cc150\n",
      "machine learning\n",
      "cc150\n"
     ]
    }
   ],
   "source": [
    "expected_features = {\n",
    "    \"favorite_books\": tf.io.VarLenFeature(dtype = tf.string),\n",
    "    \"hours\": tf.io.VarLenFeature(dtype = tf.float32),\n",
    "    \"age\": tf.io.FixedLenFeature([], dtype = tf.int64),\n",
    "}\n",
    "dataset = tf.data.TFRecordDataset([filename_fullpath])\n",
    "for serialized_example_tensor in dataset:\n",
    "    example = tf.io.parse_single_example(\n",
    "        serialized_example_tensor,\n",
    "        expected_features)\n",
    "    books = tf.sparse.to_dense(example[\"favorite_books\"],\n",
    "                               default_value=b\"\")\n",
    "    for book in books:\n",
    "        print(book.numpy().decode(\"UTF-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename_fullpath_zip = filename_fullpath + '.zip'\n",
    "options = tf.io.TFRecordOptions(compression_type = \"GZIP\")\n",
    "with tf.io.TFRecordWriter(filename_fullpath_zip, options) as writer:\n",
    "    for i in range(3):\n",
    "        writer.write(serialized_example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "machine learning\n",
      "cc150\n",
      "machine learning\n",
      "cc150\n",
      "machine learning\n",
      "cc150\n"
     ]
    }
   ],
   "source": [
    "dataset_zip = tf.data.TFRecordDataset([filename_fullpath_zip], \n",
    "                                      compression_type= \"GZIP\")\n",
    "for serialized_example_tensor in dataset_zip:\n",
    "    example = tf.io.parse_single_example(\n",
    "        serialized_example_tensor,\n",
    "        expected_features)\n",
    "    books = tf.sparse.to_dense(example[\"favorite_books\"],\n",
    "                               default_value=b\"\")\n",
    "    for book in books:\n",
    "        print(book.numpy().decode(\"UTF-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
